# -*- coding: utf-8 -*-
#!/usr/bin/env python
#
# Copyright 2007 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import webapp2
import urllib2
import json
import pprint
import time
from google.appengine.ext import db
from BeautifulSoup import BeautifulSoup
from google.appengine.ext import deferred
import crawl
from urllib2 import HTTPError

class Movie(db.Model):
    id            = db.IntegerProperty()
    title_ch      = db.StringProperty(required=True)
    title_en      = db.StringProperty()
    in_theater    = db.StringProperty()
    genre         = db.StringProperty()
    run_time      = db.StringProperty()
    director      = db.StringProperty()
    cast          = db.StringProperty()
    synopsis      = db.TextProperty()
    mini_pic_href = db.StringProperty()
    big_pic_href  = db.StringProperty()
    #times         = db.TextProperty()

class Theater(db.Model):
    id           = db.IntegerProperty()
    name         = db.StringProperty()
    address      = db.StringProperty()
    phone_number = db.StringProperty()
    hyperlink    = db.StringProperty()
    latlng       = db.GeoPtProperty()
    
class Time(db.Model):
    movie_id             = db.IntegerProperty()    
    theater_id           = db.IntegerProperty()
    theater_name         = db.StringProperty()
    theater_phone_number = db.StringProperty()
    times                = db.StringListProperty()

class MovieHandler(webapp2.RequestHandler):
    def get(self):
        
        # The Query interface prepares a query using instance methods.
        q = Movie.all()
        #q.filter("last_name =", "Smith")
        #q.filter("height <", 72)
        #q.order("-height")        
        
        # The query is not executed until results are accessed.
        jsondata = []
        results = q.fetch(50)
        for new_movie in results:
            jsondata.append( { 'title_ch' : new_movie.title_ch, 'title_en' : new_movie.title_en, 'in_theater' : new_movie.in_theater, 'genre' : new_movie.genre, 'run_time' : new_movie.run_time, 'director' : new_movie.director, 'cast' : new_movie.cast, 'synopsis' : new_movie.synopsis, 'mini_pic_href' : new_movie.mini_pic_href, 'big_pic_href' : new_movie.big_pic_href } )
            #self.response.out.write(new_movie.title_ch)
        json_str = json.dumps( { "movies" : jsondata}, ensure_ascii = False )
        #self.response.setHeader("Cache-Control", "no-cache");
        #self.response.setContentType("text/json;charset=UTF-8");
        #self.response.headers["Content-Type"] = "text/json"
        #self.response.headers.add_header("Cache-Control", "no-cache")
        self.response.out.write( json_str )                    

class TheaterHandler(webapp2.RequestHandler):
    def get(self):
        q = Theater.all()
        theater_id = self.request.get('theater_id')
        if theater_id != None:
            jsondata = []
            q.filter('id = ', int(theater_id))            
            results = q.fetch(20)
            for th in results:
                if th.latlng != None:
                    jsondata.append( { 'id' : str(th.id), 'address' : th.address, 'name' : th.name, 'phone_number' : th.phone_number , 'hyperlink' : th.hyperlink, 'lat' : th.latlng.lat, 'lng' : th.latlng.lon} )
                else:
                    jsondata.append( { 'id' : str(th.id), 'address' : th.address, 'name' : th.name, 'phone_number' : th.phone_number , 'hyperlink' : th.hyperlink } )
            json_str = json.dumps( { "theaters" : jsondata}, ensure_ascii = False )
            self.response.out.write( json_str )
        
class TimesHandler(webapp2.RequestHandler):
    def get(self):
        movie_id = self.request.get('movie_id')                
        q = Time.all()
        if movie_id != '':
            q.filter("movie_id =", int(movie_id))  
            results = q.fetch(20)
            
            jsondata = []
            for t in results:
                jsondata.append(  { 'movie_id' : t.movie_id, 'theater_id' : t.theater_id, 'theater_name' : t.theater_name, 'theater_phone_number' : t.theater_phone_number, 'times' : t.times } )
            json_str = json.dumps( { "movie_times" : jsondata}, ensure_ascii = False )
            self.response.out.write( json_str )                    
        
class CrawlMovieHandler(webapp2.RequestHandler):
    def get(self):
        '''
        HTMLParser.attrfind = re.compile(
        ur'\s*([a-zA-Z_][-.:a-zA-Z_0-9]*)(\s*=\s*'
        ur'(\'[^\']*\'|"[^"]*"|[-a-zA-Z0-9./,:;+*%?!&$\(\)_#=~@\u4e00-\u9fa5]*))?')
        '''                
        chart = urllib2.urlopen("http://tw.movies.yahoo.com/chart.html")
        soup = BeautifulSoup(chart.read())
        chart.close()

        hyperlinks = []
        bestmovie = soup.find( name = 'tr', attrs = { 'class' : 'first' } )
        anchor = bestmovie.find( name = 'a', attrs = { 'href' : True } )
        hyperlinks.append(anchor['href'])
        
        movies = soup.findAll( name = 'tr', attrs = { 'class' : 'alt' } )
        for movie in movies:
            anchor = movie.find( name = 'a', attrs = { 'href' : True } )
            hyperlinks.append(anchor['href'])
                    
        for hyperlink in hyperlinks:
            deferred.defer(crawlMovieByHyperlink, hyperlink, _countdown = 1, _queue="crawlmovie")
            #self.response.out.write(hyperlink)
            #self.response.out.write( crawl.getTimesHyperlink((hyperlink.split('=')[-1])) )
            #self.response.out.write( crawl.getTimesHyperlink((hyperlink.split('=')[-1])) )
            '''         
            attrs = crawl.crawlMovie(hyperlink)
            #self.response.out.write(attrs['times'])
            for t in attrs['times']:
                new_time  = Time( key_name = (t['movie_id'] + '_' + t['theater_id']) , movie_id = int(t['movie_id']), theater_id = int(t['theater_id']), theater_name = t['theater_name'], theater_phone_number = t['theater_phone_number'], times = t['times'] )
                self.response.out.write( new_time.theater_name )
                new_time.put()
            new_movie = Movie( key_name = attrs['title_ch'], id = int(attrs['id']), title_ch = attrs['title_ch'], title_en = attrs['title_en'], in_theater = attrs['in_theater'], genre = attrs['genre'], run_time = attrs['run_time'], director = attrs['director'], cast = attrs['cast'], synopsis = attrs['synopsis'], mini_pic_href = attrs['mini_pic_href'], big_pic_href = attrs['big_pic_href'])                
            new_movie.put()
            #time.sleep(0.2)
            '''

class CrawlTheaterHandler(webapp2.RequestHandler):
    def get(self):
        #self.response.out.write('<html><body>')
        for theater_id in range(1, 200):
            deferred.defer(crawlTheaterByTheaterId, theater_id, _countdown = 1, _queue="crawltheater")                        
            #self.response.out.write("http://tw.movie.yahoo.com/theater_result.html/id=" + theater_id)
            """
            soup = None
            for i in range(0,5):
                try:
                    theaterWeb = urllib2.urlopen("http://tw.movie.yahoo.com/theater_result.html/id=" + str(theater_id))
                    soup = BeautifulSoup(theaterWeb.read())
                    theaterWeb.close()
                    break
                except HTTPError:
                    pass
                            
            divTag = soup.find(name = 'div', attrs = {'class' : 'ymvmodule'} ).find(name = 'div', attrs = { 'class' : 'text' } )
            theater_name = unicode(divTag.find( name = 'h3' ).string)        
            if theater_name != 'None':
                phoneTag = divTag.find( name = 'h4' )
                theater_phone_number = unicode(phoneTag.string).strip(u'電話：')        
                theater_address = unicode(phoneTag.findNext( name = 'h4' ).string).strip(u'地址：')                    
                self.response.out.write(' name: ' + theater_name + '<br>')
                #self.response.out.write(' phone: ' + theater_phone_number)
                #self.response.out.write(' address: ' + theater_address)
                new_theater = Theater(key_name = theater_name, id = theater_id, name = theater_name, address = theater_address, phone_number = theater_phone_number)
                new_theater.put()
                                
                #Find GPS point
                addr2gps_url = "http://maps.googleapis.com/maps/api/geocode/json?address=" + urllib2.quote(new_theater.address.encode('utf-8')) + "&sensor=true"
                addr2gps = urllib2.urlopen( addr2gps_url )
                json_str = addr2gps.read()
                addr2gps.close()
                
                json_obj = json.loads( json_str )
                json_list = json_obj['results']
                
                if len(json_list) > 0:
                    lat = float(json_list[0]['geometry']['location']['lat'])
                    lng = float(json_list[0]['geometry']['location']['lng'])
                    self.response.out.write( str(lat) + ' ' + str(lng) )
                    new_theater.latlng = db.GeoPt(lat, lng)
                    new_theater.put()
                else:
                    new_theater.latlng = None 
                time.sleep(0.1)   
        self.response.out.write('</body></html>')
        """
        
        '''
        theaterWeb = urllib2.urlopen("http://tw.movie.yahoo.com/theater_list.html")
        soup = BeautifulSoup(theaterWeb.read())
        theaterWeb.close()
        
        groupTags = soup.findAll( name = 'div', attrs = { 'class' : 'group' } )
        for groupTag in groupTags:
            trTags = groupTag.find( name = 'tbody' ).findAll( name = 'tr' )
            for trTag in trTags:
                #Start retrieving data for a single theater
                attr = {}     
                tdTags = trTag.findAll( name = 'td' ) 
                for i in range(0, 2):
                    tdTag = tdTags[i]
                    if i == 0:
                        name = tdTag.find( name = 'a' )
                        hyperlink = name['href']
                        id = int(hyperlink.split('/')[-1].split('=')[-1])                        
                        if name != None:
                            self.response.out.write( 'name:' +  str(name.string) )
                            self.response.out.write( str(hyperlink) )
                            self.response.out.write( 'id:' + str(id) )
                            attr['name'] = unicode(name.string)
                            attr['hyperlink'] = unicode(hyperlink)
                            attr['id'] = unicode(id)
                    elif i == 1:
                        addr  = tdTag.find( text = True )
                        phone_number = tdTag.find( name = 'em' )
                        if phone_number != None:
                            self.response.out.write( 'phone:' + str(phone_number.string) )
                            attr['phone_number'] = unicode(phone_number.string)                            
                        if addr != None:
                            self.response.out.write( 'addr:' + unicode(addr) )
                            attr['address'] = unicode(addr)
                            
                            addr2gps_url = "http://maps.googleapis.com/maps/api/geocode/json?address=" + urllib2.quote(attr['address'].encode('utf-8')) + "&sensor=true"
                            #addr2gps_url = "http://maps.googleapis.com/maps/api/geocode/json?address=" + "1600+Amphitheatre+Parkway,+Mountain+View,+CA" + "&sensor=true"
                            self.response.out.write( addr2gps_url )
                            addr2gps = urllib2.urlopen( addr2gps_url )
                            json_str = addr2gps.read()
                            addr2gps.close()
                            
                            json_obj = json.loads( json_str )
                            json_list = json_obj['results']
                            
                            if len(json_list) > 0:
                                lat = float(json_list[0]['geometry']['location']['lat'])
                                lng = float(json_list[0]['geometry']['location']['lng'])
                                self.response.out.write( str(lat) + ' ' + str(lng) )
                                attr['latlng'] = db.GeoPt(lat, lng)
                            else:
                                attr['latlng'] = None
                            #except:
                            #    attr['latlng'] = None
                                #self.response.out.write( '<br>' + str(json_list[0]['geometry']['location']['lat']) + '<br>' )
                            
                            #self.response.out.write(  )
                            
                #Put into the data store
                new_theater = Theater( key_name = attr['name'], id = attr['id'], name = attr['name'], address = attr['address'], phone_number = attr['phone_number'], hyperlink = attr['hyperlink'], latlng = attr['latlng'])
                key = new_theater.put()
                time.sleep(0.2)
                #deferred.defer(fetchTheaterGeoPtFromGoogle, key, _countdown = 1, _queue="geocodingqueue")
        '''                                     

def crawlMovieByHyperlink(hyperlink):
    attrs = crawl.crawlMovie(hyperlink)    
    for t in attrs['times']:
        new_time  = Time( key_name = (t['movie_id'] + '_' + t['theater_id']) , movie_id = int(t['movie_id']), theater_id = int(t['theater_id']), theater_name = t['theater_name'], theater_phone_number = t['theater_phone_number'], times = t['times'] )        
        new_time.put()
    new_movie = Movie( key_name = attrs['title_ch'], id = int(attrs['id']), title_ch = attrs['title_ch'], title_en = attrs['title_en'], in_theater = attrs['in_theater'], genre = attrs['genre'], run_time = attrs['run_time'], director = attrs['director'], cast = attrs['cast'], synopsis = attrs['synopsis'], mini_pic_href = attrs['mini_pic_href'], big_pic_href = attrs['big_pic_href'])                
    new_movie.put()

def crawlTheaterByTheaterId(theater_id, max_trial = 5):
    soup = None
    for i in range(0, max_trial):
        try:
            theaterWeb = urllib2.urlopen("http://tw.movie.yahoo.com/theater_result.html/id=" + str(theater_id))
            soup = BeautifulSoup(theaterWeb.read())
            theaterWeb.close()
            break
        except HTTPError:
            pass
                    
    divTag = soup.find(name = 'div', attrs = {'class' : 'ymvmodule'} ).find(name = 'div', attrs = { 'class' : 'text' } )
    theater_name = unicode(divTag.find( name = 'h3' ).string)        
    if theater_name != 'None':
        phoneTag = divTag.find( name = 'h4' )
        theater_phone_number = unicode(phoneTag.string).strip(u'電話：')        
        theater_address = unicode(phoneTag.findNext( name = 'h4' ).string).strip(u'地址：')                            
        #self.response.out.write(' phone: ' + theater_phone_number)
        #self.response.out.write(' address: ' + theater_address)
        new_theater = Theater(key_name = theater_name, id = theater_id, name = theater_name, address = theater_address, phone_number = theater_phone_number)
        theater_key = new_theater.put()
        
        deferred.defer(fetchTheaterGeoPtByKeyFromGoogle, theater_key, _countdown = 1, _queue="geocodingqueue")
        '''                
        #Find GPS point
        addr2gps_url = "http://maps.googleapis.com/maps/api/geocode/json?address=" + urllib2.quote(new_theater.address.encode('utf-8')) + "&sensor=true"
        addr2gps = urllib2.urlopen( addr2gps_url )
        json_str = addr2gps.read()
        addr2gps.close()
        
        json_obj = json.loads( json_str )
        json_list = json_obj['results']
        
        if len(json_list) > 0:
            lat = float(json_list[0]['geometry']['location']['lat'])
            lng = float(json_list[0]['geometry']['location']['lng'])
            self.response.out.write( str(lat) + ' ' + str(lng) )
            new_theater.latlng = db.GeoPt(lat, lng)
            new_theater.put()
        else:
            new_theater.latlng = None         
        '''   
                       
def fetchTheaterGeoPtByKeyFromGoogle(self, theater_key):
    theater = db.get(theater_key) 
    addr = theater.address
    addr2gps_url = "http://maps.googleapis.com/maps/api/geocode/json?address=" + urllib2.quote(addr.encode('utf-8')) + "&sensor=true"
    #addr2gps_url = "http://maps.googleapis.com/maps/api/geocode/json?address=" + "1600+Amphitheatre+Parkway,+Mountain+View,+CA" + "&sensor=true"
    self.response.out.write( addr2gps_url )
    addr2gps = urllib2.urlopen( addr2gps_url )
    json_str = addr2gps.read()
    addr2gps.close()
    
    json_obj = json.loads( json_str )
    json_list = json_obj['results']
    
    if len(json_list) > 0:
        lat = float(json_list[0]['geometry']['location']['lat'])
        lng = float(json_list[0]['geometry']['location']['lng'])
        self.response.out.write( str(lat) + ' ' + str(lng) )
        theater.latlng = db.GeoPt(lat, lng)
        theater.put()

app = webapp2.WSGIApplication([('/movie/', MovieHandler), ('/theater/.*', TheaterHandler), ('/time/.*', TimesHandler), ('/crawl/movie/', CrawlMovieHandler), ('/crawl/theater/', CrawlTheaterHandler)],
                              debug=True)
